package cc.yihy.spider.core;

import cc.yihy.spider.SpiderConfigurationProperties;
import cc.yihy.spider.util.MarkdownUtils;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.RedisPriorityScheduler;

import javax.annotation.PostConstruct;
import java.util.List;

/**
 * @Package: cc.yihy
 * @date: 2017/11/21 14:01
 * @author: Yihy
 * @version: V1.0
 * @Description:
 */
@Component
public class JianShuPageProcesser implements PageProcessor {

    @Autowired
    private SpiderConfigurationProperties properties;

    public static final String URL_POST = "http://www\\.jianshu\\.com/p/[0-9a-zA-Z]{3,20}";
    public static final String URL_POST_S = "http://www.jianshu.com/p/%s";
    public static final String URL_LIST = "(http://www\\.jianshu\\.com/u/[0-9a-zA-Z]{3,20})|(http://www\\.jianshu\\.com/nb/[0-9a-zA-Z]{3,20})|(http://www\\.jianshu\\.com/c/[0-9a-zA-Z]{3,20})";
    public static final String URL_USER = "(http://www\\.jianshu\\.com/u/[0-9a-zA-Z]{3,20})|(http://www\\.jianshu\\.com/users/[0-9a-zA-Z]{3,20})";
    public static final String URL_USER1 = "http://www\\.jianshu\\.com/mobile/users/[0-9]{1,20}/public_notes";
    public static final String URL_USER2 = "http://www.jianshu.com/mobile/users/%s/public_notes?page=%s&count=50&order_by=shared_at";
    public static final String URL_COMM = "http://www.jianshu.com/notes/[0-9]{1,20}/recommendations";
    private Site site = Site.me()
            .setDomain("www.jianshu.com")
            .setSleepTime(10)
            .setUserAgent(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");



    @PostConstruct
    public void init() {
        site = Site.me()
                .setDomain(properties.getDomain())
                .setSleepTime(properties.getSleepTime())
                .setUserAgent(properties.getUserAgent());

    }
    @Override
    public void process(Page page) {
        List<String> links = page.getHtml().links().regex(URL_POST).all();
        page.addTargetRequests(links);

        //列表页
        JsonParser jsonParser1 = new JsonParser();
        if (page.getUrl().regex(URL_COMM).match()) {

            String s = page.getRawText();
            if (s == null || s.isEmpty()) return;
            if (s.trim().length() > 2) {
                JsonArray asJsonArray = jsonParser1.parse(s).getAsJsonArray();
                for (JsonElement element : asJsonArray) {
                    String slug = element.getAsJsonObject().get("slug").getAsString();
                    page.addTargetRequest(String.format(URL_POST_S, slug));
                }

            }
            page.setSkip(true);
        } else if (page.getUrl().regex(URL_USER1).match()) {
            page.setSkip(true);
            String s = page.getRawText();
            if (s == null || s.isEmpty()) return;
            if (s.trim().length() > 2) {
                JsonArray asJsonArray = jsonParser1.parse(s).getAsJsonArray();
                for (JsonElement element : asJsonArray) {
                    String slug = element.getAsJsonObject().getAsJsonObject("object").get("data").getAsJsonObject().get("slug").getAsString();
                    page.addTargetRequest(String.format(URL_POST_S, slug));
                }

                page.addTargetRequest(new Request(nextPage(page.getUrl().get())).addHeader("Accept", "application/json"));


            }

        } else if (page.getUrl().regex(URL_POST).match()) {

            page.putField("title", page.getHtml().xpath("//h1[@class='title']/text()").get());
            page.putField("author", page.getHtml().$("div.follow-detail a.title", "text").get());
            page.putField("tag", page.getHtml().$("a.notebook > span", "text").get());
            page.putField("publishTime", page.getHtml().$(".publish-time", "text").toString());
            page.putField("content", page.getHtml().$("div.show-content").get());
            page.putField("markdown", MarkdownUtils.convertHtmlToMarkDown((String) page.getResultItems().get("content")));
            page.putField("jsonInfo", page.getHtml().xpath("//script[@data-name='page-data']/html()").get());
            String s = page.getHtml().xpath("//div[@data-vcomp='recommended-notes']/@data-note-id").get();
//<div data-vcomp="recommended-notes" data-lazy="1.5" data-note-id="20013900"></div>
            JsonParser jsonParser = jsonParser1;
            JsonElement jsonInfo = jsonParser.parse((String) page.getResultItems().get("jsonInfo"));
            int asInt = jsonInfo.getAsJsonObject().getAsJsonObject("note").get("id").getAsInt();
            int userId = jsonInfo.getAsJsonObject().getAsJsonObject("note").get("user_id").getAsInt();
            page.putField("id", asInt);
            page.addTargetRequest(new Request("http://www.jianshu.com/notes/" + s + "/recommendations").addHeader("Accept", "application/json"));
            page.addTargetRequest(new Request(String.format(URL_USER2, userId, "1")).addHeader("Accept", "application/json"));

        } else {
            page.setSkip(true);
        }
        page.addTargetRequests(page.getHtml().links().regex(URL_POST).all());
        page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());

    }

    @Override
    public Site getSite() {
        return site;
    }




    public String nextPage(String url) {
        String substring = url.substring(url.indexOf("=") + 1, url.indexOf("&"));

        Integer integer = Integer.valueOf(substring) + 1;

        return url.replace("=" + substring + "&", "=" + integer + "&");
    }
}
